package com.example.myFirstWebmagic.hourAirReport.processor;

import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;

import com.example.myFirstWebmagic.dailyAirReport.entity.AirQualityReportPK;
import com.example.myFirstWebmagic.hourAirReport.entity.EnHourAirQualityReport;
import com.example.myFirstWebmagic.pipeline.ConsolePipelineDemo;
import com.example.myFirstWebmagic.util.DateUtil;
import com.example.myFirstWebmagic.util.UrlUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 
 * ClassName: NewAirHourProcess 
 * @Description: 空气质量小时报
 * @author xiongyao
 * @date 2017年3月5日
 */
public class NewAirHourProcess implements PageProcessor{

	private Site site = Site.me().setRetryTimes(10).setSleepTime(5000);
	private String date;
	private List<String> targetUrls=new ArrayList<String>();
	 //所有的地市
    private static List<String> cities=new ArrayList<String>();
	
	private List<EnHourAirQualityReport> qualityReports=new ArrayList<EnHourAirQualityReport>();
	
	private static final String TARGETURL="http://datacenter.mep.gov.cn/index!environmentAirHourDetail.action?cityName=cityName&searchTime=searchTime";
	
	@Override
	public void process(Page page) {
		//在首页抓取所有的地市连接
		if(page.getRequest().getUrl().equals("http://datacenter.mep.gov.cn/index!environmentAirHourDetailCity.action")){
			cities=page.getHtml().xpath("//div[@class='aqi_list']//span[@class='clickcss']/text()").all();
			
			for(String city:cities){
				String url=UrlUtil.replaceUrlParamValue(TARGETURL, "cityName", city);
				url=UrlUtil.replaceUrlParamValue(url, "searchTime", DateUtil.getToday());
				targetUrls.add(url);
			}
			
			//targetUrls=page.getHtml().xpath("/html/body/table/tbody//a/@href").all();
			//如果没有设置抓取时间，则默认为最新时间
			if(StringUtils.isEmpty(this.getDate())){
				page.addTargetRequests(targetUrls);
			}else{
				//否则抓取指定日期数据
				for(String url:targetUrls){
					url=UrlUtil.replaceUrlParamValue(url, "searchTime", this.getDate());
					page.addTargetRequest(url);
				}
			}
		}else{
			//日期
			List<String> dates= page.getHtml().xpath("/html/body/div[2]/div[3]/table/tbody/tr[2]/td/table/tbody//tr/td[2]/text()").all();
			//城市
			List<String> citys= page.getHtml().xpath("/html/body/div[2]/div[3]/table/tbody/tr[2]/td/table/tbody//tr/td[3]/text()").all();
			//aqi指数
			List<String> aqis= page.getHtml().xpath("/html/body/div[2]/div[3]/table/tbody/tr[2]/td/table/tbody//tr/td[4]/text()").all();
			//空气质量级别
			List<String> airLevels= page.getHtml().xpath("/html/body/div[2]/div[3]/table/tbody/tr[2]/td/table/tbody//tr/td[5]/text()").all();
			//污染物
			List<String> populates= page.getHtml().xpath("/html/body/div[2]/div[3]/table/tbody/tr[2]/td/table/tbody//tr/td[6]/text()").all();
			
			//Map<AirQualityReportPK, EnHourAirQualityReport> qualityReports=new HashMap<AirQualityReportPK, EnHourAirQualityReport>();
			
			
			
			for(String date:dates){
				int index=dates.indexOf(date);
				AirQualityReportPK airQualityReportPK=new AirQualityReportPK(citys.get(index), date);
				EnHourAirQualityReport airQualityReport=new EnHourAirQualityReport(airQualityReportPK, aqis.get(index), airLevels.get(index), populates.get(index));
				qualityReports.add(airQualityReport);
			}
			page.putField("objectList", qualityReports);
		}
		
	}

	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return this.site;
	}
	
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String [] startUrl=new String[]{"http://datacenter.mep.gov.cn/index!environmentAirHourDetailCity.action"};
		// TODO Auto-generated method stub
		Spider.create(new NewAirHourProcess())
        //从"http://datacenter.mep.gov.cn/index!getCityMain.action"开始抓
        .addUrl(startUrl)
        .addPipeline(new ConsolePipelineDemo())
        //开启5个线程抓取
        .thread(5)
        //启动爬虫
        .run();
	}

	
	public String getDate() {
		return date;
	}

	public void setDate(String date) {
		this.date = date;
	}
}
